#drafted by Jeremy VanDerWal ( jjvanderwal@gmail.com ... www.jjvanderwal.com )
#GNU General Public License .. feel free to use / distribute ... no warranties

################################################################################
#prepare for loops

#define if this is to be run @ UQ
UQ=FALSE

#define years of interest and variables of interest
years = 2011:2012
vois = c('rain','tmin','tmax')

#define some directories
base.dir = ifelse (UQ,'/home/uqvdwj/','/home/jc148322/') 
base.data.dir = paste(base.dir,'Climate/AWAP.direct.download/',sep='')
out.dir = paste(base.dir,'Climate/AWAP.direct.download/summaries/Oz/monthly.csv/',sep=''); dir.create(out.dir,recursive=TRUE)
pbs.dir = paste(base.dir,'tmp.pbs/',sep='')

#define the mask postions
pos.file = paste('/home/jc165798/','Climate/OZmask/mask.pnts.csv',sep='')
###########################################################################################
###########################################################################################
#extract the annual data into ANNUAL csv files

setwd(pbs.dir)
for (year in years) {
	for (voi in vois) {
		##create the sh file
		zz = file(paste(voi,year,'tmp.sh',sep=''),'w')
			cat('##################################\n',file=zz)
			cat('#!/bin/bash\n',file=zz)
			cat('source /etc/profile \n',file=zz)
			cat('module load R \n',file=zz) #load the necessary module				
			cat('cd $PBS_O_WORKDIR\n',file=zz)
			cat('R CMD BATCH --no-save ',voi,year,'tmp.R\n',sep='',file=zz)
			cat('##################################\n',file=zz)
		close(zz)
		#create the R summarizaing script
		zz = file(paste(voi,year,'tmp.R',sep=''),'w')
			cat('library(SDMTools)\n',file=zz)
			cat('#define directories\n',sep='',file=zz)
			cat('data.dir = "',base.data.dir,voi,'/monthly/"\n',sep='',file=zz)
			cat('out.dir = "',out.dir,'"\n',sep='',file=zz)
			cat('#define the files to use\n',sep='',file=zz)
			cat('tfiles = list.files(data.dir,pattern=as.character(',year,')) #list all files for year of interest\n',sep='',file=zz)
			cat('if (any(substr(tfiles,1,4)!=',year,')) tfiles = tfiles[-which(substr(tfiles,1,4)!=',year,')] #ensure all files start with the year\n',sep='',file=zz)
			cat('#read in the locations we want to extract data for\n',sep='',file=zz)
			cat('pos = read.csv("',pos.file,'",as.is=T)\n',sep='',file=zz)
			cat('#cycle through the files and extract the data\n',file=zz)
			cat('for (tfile in tfiles) {\n',file=zz)
			cat('    tasc = read.asc.gz(paste(data.dir,tfile,sep=""))\n',file=zz)
			cat('    tname = gsub("\\\\.asc.gz","",tfile); tname = paste("Ym",tname,sep="")\n',file=zz)
			cat('    pos[tname] = extract.data(cbind(pos$lon,pos$lat),tasc)\n',file=zz)
			cat('}\n',file=zz)
			cat('#write out the data\n',file=zz)
			cat('write.csv(pos,file="',out.dir,voi,year,'tmp.csv",row.names=FALSE)\n',sep='',file=zz)	
		close(zz)
		#submit the job
		tt = ifelse(UQ,paste('qsub -l select=1:ncpus=2:NodeType=medium -l walltime=100:00:00 -A q1086 ',voi,year,'tmp.sh',sep=''),paste('qsub ',voi,year,'tmp.sh',sep=''))
		system(tt)
	}
}
#check for all outputs
sh.files = list.files(,pattern='tmp.R'); sh.files = sh.files[-grep('Rout',sh.files)]; sh.files=gsub('.R','',sh.files)
Rout.files = list.files(,pattern='tmp.Rout'); Rout.files = gsub('.Rout','',Rout.files)
t3 = setdiff(sh.files,Rout.files)
rerun.files = paste(t3,'.sh',sep='')
for (tfile in rerun.files) {
	tt = ifelse(UQ,paste('qsub -l select=1:ncpus=2:NodeType=medium -l walltime=100:00:00 -A q1086 ',tfile,sep=''),paste('qsub ',tfile,sep=''))
	system(tt);
} 

###########################################################################################
###########################################################################################
#bring together the annual csv files
setwd(pbs.dir)
#clean up the pbs directory
system(paste('rm -rf ',pbs.dir,'*',sep=''))
#cycle through and create the composite data
for (voi in vois) {
	##create the sh file
	zz = file(paste(voi,'tmp.sh',sep=''),'w')
		cat('##################################\n',file=zz)
		cat('#!/bin/bash\n',file=zz)
		cat('source /etc/profile \n',file=zz)
		cat('module load R \n',file=zz) #load the necessary module				
		cat('cd $PBS_O_WORKDIR\n',file=zz)
		cat('R CMD BATCH --no-save ',voi,'tmp.R\n',sep='',file=zz)
		cat('##################################\n',file=zz)
	close(zz)
	#create the R summarizaing script
	zz = file(paste(voi,'tmp.R',sep=''),'w')
		cat('#define directories\n',sep='',file=zz)
		cat('out.dir = "',out.dir,'"; setwd(out.dir)\n',sep='',file=zz)
		cat('#define the files to use\n',sep='',file=zz)
		cat('tfiles = list.files(,pattern=as.character("',voi,'")) #list all files for year of interest\n',sep='',file=zz)
		cat('#read in the locations we want to extract data for\n',sep='',file=zz)
		cat('pos = read.csv("',pos.file,'",as.is=T)\n',sep='',file=zz)
		cat('#cycle through and create the dataframe\n',sep='',file=zz)
		cat('colnames = NULL\n',sep='',file=zz)
		cat('for (tfile in tfiles) {\n',file=zz)
		cat('    colname = gsub("tmp.csv","",tfile); colname = gsub("',voi,'","",colname)\n',sep='',file=zz)
		cat('    colnames = c(colnames,paste("Ym",colname,sprintf("%02i",1:12),sep=""))\n',sep='',file=zz)
		cat('}\n',file=zz)
		cat('pos = data.frame(pos,matrix(NA,nr=nrow(pos),nc=length(colnames),dimnames=list(1:nrow(pos),colnames)))\n',file=zz)
		cat('#cycle through the files and extract the data\n',file=zz)
		cat('for (tfile in tfiles) {\n',file=zz)
		cat('    cat(tfile,"\\n")\n',file=zz)
		cat('    tdata = read.csv(tfile)\n',file=zz)
		cat('    for(ii in 3:14) { pos[names(tdata)[ii]] = tdata[,ii] } #append the data\n',file=zz)
		cat('    unlink(tfile) #remove the file\n',file=zz)
		cat('}\n',file=zz)
		cat('#write out the data\n',file=zz)
		cat('write.csv(pos,file="',voi,min(years),max(years),'.csv",row.names=FALSE)\n',sep='',file=zz)	
	close(zz)
	#submit the job
	tt = ifelse(UQ,paste('qsub -l select=1:ncpus=8:NodeType=medium -l walltime=100:00:00 -A q1086 ',voi,'tmp.sh',sep=''),paste('qsub ',voi,'tmp.sh',sep=''))
	system(tt)
}

###########################################################################################
###########################################################################################
#create Tmean

#bring together the annual csv files
setwd(pbs.dir)
#clean up the pbs directory
system(paste('rm -rf ',pbs.dir,'*',sep=''))

##create the sh file
zz = file('tmean.sh','w')
	cat('##################################\n',file=zz)
	cat('#!/bin/bash\n',file=zz)
	cat('source /etc/profile \n',file=zz)
	cat('module load R \n',file=zz) #load the necessary module				
	cat('cd $PBS_O_WORKDIR\n',file=zz)
	cat('R CMD BATCH --no-save tmean.R\n',sep='',file=zz)
	cat('##################################\n',file=zz)
close(zz)
#create the R summarizaing script
zz = file('tmean.R','w')
	cat('#define directories\n',sep='',file=zz)
	cat('out.dir = "',out.dir,'"; setwd(out.dir)\n',sep='',file=zz)
	cat('#define the files to use\n',sep='',file=zz)
	cat('tmin = as.matrix(read.csv(file="tmin',min(years),max(years),'.csv"))\n',sep='',file=zz)
	cat('tmax = as.matrix(read.csv(file="tmax',min(years),max(years),'.csv"))\n',sep='',file=zz)
	cat('tmean = (tmin + tmax) /2\n',file=zz)
	cat('#write out the data\n',file=zz)
	cat('write.csv(tmean,file="tmean',min(years),max(years),'.csv",row.names=FALSE)\n',sep='',file=zz)	
close(zz)
#submit the job
tt = ifelse(UQ,paste('qsub -l select=1:ncpus=8:NodeType=medium -l walltime=100:00:00 -A q1086 tmean.sh',sep=''),paste('qsub tmean.sh',sep=''))
system(tt)
